#
# fusion.crawler.crawler
#
# Copyright 2007 Helsinki Institute for Information Technology
# and the authors.
#
# Authors: Ken Rimey <rimey@hiit.fi>
#

# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import time
import logging

import simplejson

from fusion.db.db import DB

class Crawler(object):
    def __init__(self, path):
        self.path = path
        self.db = None

    def loop(self, interval=2.0):
        self.init()
        try:
            mark = 0
            while True:
                mark = self.crawl(mark)
                time.sleep(interval)
        finally:
            self.close()

    def init(self):
        self.db = DB(self.path)

    def close(self):
        if self.db is not None:
            self.db.close()

    def crawl(self, last_mark=0):
        referenced = set()

        for c, k in self.db.query_all_collections_and_keys(last_mark):
            if k.startswith('link/'):
                v = self.db.get(c, k)
                if v:
                    profile = k.split('/', 1)[1]
                    referenced.add(profile)
            elif k.startswith('guestbook/') or k.startswith('member_info/'):
                v = self.db.get(c, k)
                if v:
                    try:
                        info = simplejson.loads(v)
                    except ValueError:
                        continue

                    profile = info.get('profile')
                    if profile:
                        referenced.add(profile)

        for c in referenced:
            self.require(c)

        return self.db.get_last_mark()

    def require(self, c):
        if isinstance(c, (str, unicode)) and len(c) == 40 \
                and all((x in '0123456789abcdef') for x in c):
            if not self.db.collection_exists(c):
                logging.info('Crawler importing collection: %s' % c)
                self.db.import_collection(c)
